{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upstash Vector Store\n",
    "\n",
    "We're going to look at how to use LlamaIndex to interface with Upstash Vector!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: llama-index in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (0.9.35)\n",
      "Requirement already satisfied: upstash-vector in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (0.2.0)\n",
      "Requirement already satisfied: SQLAlchemy>=1.4.49 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index) (2.0.25)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (3.9.1)\n",
      "Requirement already satisfied: dataclasses-json in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (0.6.3)\n",
      "Requirement already satisfied: deprecated>=1.2.9.3 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (1.2.14)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (2023.12.2)\n",
      "Requirement already satisfied: httpx in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (0.25.2)\n",
      "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (1.6.0)\n",
      "Requirement already satisfied: networkx>=3.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (3.1)\n",
      "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (3.8.1)\n",
      "Requirement already satisfied: numpy in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (1.24.4)\n",
      "Requirement already satisfied: openai>=1.1.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (1.9.0)\n",
      "Requirement already satisfied: pandas in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (2.0.3)\n",
      "Requirement already satisfied: requests>=2.31.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (8.2.3)\n",
      "Requirement already satisfied: tiktoken>=0.3.3 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (0.5.2)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (4.9.0)\n",
      "Requirement already satisfied: typing-inspect>=0.8.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from llama-index) (0.9.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (23.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.9.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.4.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (1.3.1)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index) (4.0.3)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from deprecated>=1.2.9.3->llama-index) (1.16.0)\n",
      "Requirement already satisfied: anyio in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpx->llama-index) (4.2.0)\n",
      "Requirement already satisfied: certifi in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpx->llama-index) (2023.11.17)\n",
      "Requirement already satisfied: httpcore==1.* in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpx->llama-index) (1.0.2)\n",
      "Requirement already satisfied: idna in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpx->llama-index) (3.6)\n",
      "Requirement already satisfied: sniffio in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpx->llama-index) (1.3.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from httpcore==1.*->httpx->llama-index) (0.14.0)\n",
      "Requirement already satisfied: click in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from nltk<4.0.0,>=3.8.1->llama-index) (8.1.7)\n",
      "Requirement already satisfied: joblib in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from nltk<4.0.0,>=3.8.1->llama-index) (1.3.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from nltk<4.0.0,>=3.8.1->llama-index) (2023.12.25)\n",
      "Requirement already satisfied: tqdm in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from nltk<4.0.0,>=3.8.1->llama-index) (4.66.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from openai>=1.1.0->llama-index) (1.9.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from openai>=1.1.0->llama-index) (1.10.14)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from requests>=2.31.0->llama-index) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from requests>=2.31.0->llama-index) (1.26.18)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from SQLAlchemy>=1.4.49->SQLAlchemy[asyncio]>=1.4.49->llama-index) (3.0.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from typing-inspect>=0.8.0->llama-index) (1.0.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from dataclasses-json->llama-index) (3.20.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from pandas->llama-index) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from pandas->llama-index) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from pandas->llama-index) (2023.4)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from anyio->httpx->llama-index) (1.2.0)\n",
      "Requirement already satisfied: packaging>=17.0 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from marshmallow<4.0.0,>=3.18.0->dataclasses-json->llama-index) (23.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/meshan/.cache/pypoetry/virtualenvs/llama-index-g12BQvjh-py3.8/lib/python3.8/site-packages (from python-dateutil>=2.8.2->pandas->llama-index) (1.16.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install llama-index upstash-vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
    "from llama_index.vector_stores import UpstashVectorStore\n",
    "from llama_index.storage.storage_context import StorageContext\n",
    "import textwrap\n",
    "import openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup the OpenAI API\n",
    "openai.api_key = \"sk-...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-02-03 20:04:25--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 75042 (73K) [text/plain]\n",
      "Saving to: ‘data/paul_graham/paul_graham_essay.txt’\n",
      "\n",
      "data/paul_graham/pa 100%[===================>]  73.28K  --.-KB/s    in 0.01s   \n",
      "\n",
      "2024-02-03 20:04:25 (5.96 MB/s) - ‘data/paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Download data\n",
    "! mkdir -p 'data/paul_graham/'\n",
    "! wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we can load the documents using the LlamaIndex SimpleDirectoryReader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Documents: 1\n"
     ]
    }
   ],
   "source": [
    "documents = SimpleDirectoryReader(\"./data/paul_graham/\").load_data()\n",
    "\n",
    "print(\"# Documents:\", len(documents))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To create an index on Upstash, visit https://console.upstash.com/vector, create an index with 1536 dimensions and `Cosine` distance metric. Copy the URL and token below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_store = UpstashVectorStore(url=\"https://...\", token=\"...\")\n",
    "\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents, storage_context=storage_context\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we've successfully created an index and populated it with vectors from the essay! The data will take a second to index and then it'll be ready for querying."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The author learned that the study of philosophy in college did not live up to their expectations.\n",
      "They found that other fields took up most of the space of ideas, leaving little room for what they\n",
      "perceived as the ultimate truths that philosophy was supposed to explore. As a result, they decided\n",
      "to switch to studying AI.\n",
      "\n",
      "\n",
      "The author's opinion on startups is that they are in need of help and support, especially in the\n",
      "beginning stages. The author believes that founders of startups are often helpless and face various\n",
      "challenges, such as getting incorporated and understanding the intricacies of running a company. The\n",
      "author's investment firm, Y Combinator, aims to provide seed funding and comprehensive support to\n",
      "startups, offering them the guidance and resources they need to succeed.\n"
     ]
    }
   ],
   "source": [
    "query_engine = index.as_query_engine()\n",
    "res1 = query_engine.query(\"What did the author learn?\")\n",
    "print(textwrap.fill(str(res1), 100))\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "res2 = query_engine.query(\"What is the author's opinion on startups?\")\n",
    "print(textwrap.fill(str(res2), 100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
